{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch 0\n",
      "Hypothesis [[ 0.48712057  0.51287943]\n",
      " [ 0.3380821   0.66191792]\n",
      " [ 0.65063184  0.34936813]\n",
      " [ 0.5031724   0.49682763]]\n",
      "w1=[[-0.79593647  0.93947881]\n",
      " [ 0.68854761 -0.89423609]]\n",
      "b1=[-0.00733338  0.00893857]\n",
      "w2=[[-0.79084051  0.93289936]\n",
      " [ 0.69278169 -0.8986907 ]]\n",
      "b2=[ 0.00394399 -0.00394398]\n",
      "cost (ce)=2.87031\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXYAAAD8CAYAAABjAo9vAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAF7pJREFUeJztnVGIpWd9xp//zO5SFlO8yLZKdmeOVLogttTOIpZcKIlN\n0zGYFgptouLixbTTChEi1pibUhApltiCBUnsRcsZEGmVglTSBEvJTcXZNLFNorK4uyap0g2FVshF\nye6/F+ck2c1+75nznXm/7/2/z/s8cNjMN+ec3/f8j7wz/s6Z7zV3h6IoisKTtdInoCiKouSNFnZF\nURSyaGFXFEUhixZ2RVEUsmhhVxRFIYsWdkVRFLJoYVcURSGLFnZFURSyaGFXFEUhy5ES0Jtvvtkn\nk0kJtKIoSrU5d+7cS+5+4qD7FVnYJ5MJ9vf3S6AVRVGqjZldWuZ+UjGKoihk0cKuKIpCFi3siqIo\nZNHCriiKQhYt7IqiKGSpZ2Hf2wMmE2Btbfbv3t7i46s8JtdxdjZ7P81W7BWfaxXEIHH30W9bW1ve\nK9Op+/Hj7sDrt+PH3Xd3u49Pp/0fk+s4O5u9n2Yr9orPNd19ojdiOu23FALYX2aNtRJb4505c8Z7\nfY59MgEudXx8c30duHLlxuObm7N/+zwm13F2Nns/zVbsFZ9rsv48Ll052QuxuQlcvHjj8VTM7Jy7\nnznwflUs7Gtrsx9yy8Zs9m+BbvRs9n4l2ez9yNlruALvsNuLEGbA1avLM5Zd2Otw7Bsb3cfX19P3\n7/uYXMfZ2ez9NFuxl3iuPdyDCS5gDVcwwQXs4R5srP9nb0Tq+GFTx8K+vd19/PTp9P37PibXcXY2\nez/NVuwDHrN3+k+xg0dwCRM41nAJE+zgEbz9tPVGpI4fNnWoGDn2OGz2fpqt2Ac8JuXSV0HIscux\nx2Cz9yvJZu9Hwk659FUQcuxdIXJ21bDZ+2m2Yl/zmD4ufRWEHHtXSJxdVWz2fpqt2PP0demr1JNj\nl2OPwWbvp9mKPU9fly7HLsdeL5u9X0k2e7/K2H1d+goIOfbOVOjsqmez99Nsm2TncOly7H0jxx6H\nzd5Ps22Ovbc9zeLS5djl2Otls/fTbJtjT3Cx2Msqxy7HHoPN3q8km71fUPYarhZ7WeXYuxLY2dGy\n2ftptrTsLo+OjY2iL6sce1eCOjtqNns/zZaSnfpM+t72tOjLKscuxx6Dzd5Ps6Vkr3KtdDn2GXAd\nwD6AF939rkX3lWOvmM3erySbvV9B9irXSs+V2h37fQCey/h8r0eOPQ6bvZ9mWz0717XSm3fsZnYS\nwAcAfDnH890QOfY4bPZ+mm3V7JzXSm/esZvZ3wH4HICbAHwyu4qRY4/DZu+n2VbNznmt9KYdu5nd\nBWDb3f/QzN6HxMJuZjsAdgBgY2Nj61JXy1Tk2OOw2fuVZLP3G4Gd81rpuVKrY78VwAfN7CKArwC4\nzcymb7yTuz/s7mfc/cyJEyf6EeTY47DZ+2m21bCHvlZ6047d3R9w95PuPgHwewC+5e4fPvSZXRs5\n9jhs9n6abRXsMa6V3rxjf+3JFqiYayPHXjGbvZ9mWwV7jGulN+3YV4k+x14xm71fSTZ7v4zsMa6V\nniu1OvbhI8ceh83eT7MNxy51rfSmHfsokWOPw2bvp9mGYudy6VFf1ioc+7KRY6+Yzd5Psw3FzuXS\no76scuxy7DHY7P1Kstn7rcDO5dJXQI/CkGPvSoW+sHo2ez/NNhQ7l0uP+rIO5djh7qPftra2vFem\nU/djx9xnP/Rmt2PH3Hd3u49Pp/0fk+s4O5u9n2ZbjD09etY3ccENV3wTF3x69KxPd5+ovvYixnTa\nbykEsL/MGntkoJ8X+ePe/XXq+CqPyXWcnT0Go1X2GIyA7D18CDv2u3h5viRdwgQ79gg+iiMUtQ/6\nXu7U4dj15mkcNns/zbYIe+gNpaO+rHrztM95Bn0jiILN3q8km73fAvbQG0ovQAMo97LqzdOuVPZG\nEAWbvZ9mOyi71IbSUV9W/YFSVyr7YwsKNns/zXYwRskNpaO+rPoDJTn2GGz2fprtYIySG0pHfVnl\n2OXYY7DZ+5Vkk/cruaF0KqVfVjn2rjTsKoux2ftptoMxSm4oHfVllWPvSqOuMqwwZOin2Q7G2D59\nvhQ67Msqxy7HHoPN3k+zHYwhx979PTn2ZcMu7Uqy2fuVZJP3k2Pv/p4c+xvTqKsMKwwZ+mm2WRh9\nNscgqt2bIcfelUZdZVhhyNBPsz30c/XdHIOkthy7HHvFbPZ+mu2hn6vv5hgkteXY5dgrZrP3K8km\n6dd3cwyS2isx5Ni70oCrDMdm76fZ9nquHBtNV1g7G0OOvSvkrjIkm72fZrv0Y3JtNF1ZbTn2VOTY\nK2az99Nsl35Mro2mK6stx56KHHvFbPZ+JdmV9cu10XRltbMy5Ni70qq0iyoMGfpptp2PyeHSK6w9\nOEOOvSutSruowpChn2Z7Q3K59Mpqy7H3jRx7xWz2fprtDd/K5dIrqy3H3jdy7BWz2fuVZAftl8ul\nr4AePKVfVjn2rrQq7aIKQ4Z+jc92SJceuHYxhhx7V1qVdlGFIUO/hme7tz0d1KUHrS3H3vkEZqcA\n/C2AnwfgAB52979c9Bg59orZ7P0anu0EF2lHG/VlDevYzeytAN7q7k+a2U0AzgH4LXd/NvUYOfaK\n2ez9SrIL91vDVdrRlmRX6djd/cfu/uT8v38K4DkAtxz2ea+LHHscNnu/Bmbb5dGxsUE92qgvaxWO\n3cwmAN4F4Ns5n1eOPRCbvR/5bFOfSd/bnlKPNurLGtaxv/ZEZm8C8C8APuvuX+v4/g6AHQDY2NjY\nutQlnFKRY4/DZu9HPttW9x2N+rKGdexz2FEA3wDwqLs/dND95dgrZrP3K8nWvqOU7Codu5kZgL8G\n8Nwyi/pKkWOPw2bvRzRb7Tsag12rY78VwEcA3GZmT81vec2RHHscNns/ktlq39E47Kode5/oc+wV\ns9n7kcxW+47GYVfr2PtGjr1iNnu/kmztO0rJrtKxjxI59jhs9n4Vzlb7jsZm1+rYh48cexw2e7/K\nZqt9R+Oz5dhTkWOPw2bvV9lste9ofLYceypy7HHY7P1KsrXvKCVbjj0VOfY4bPZ+gWerfUfrZMux\npyLHHofN3i/obLXvaL1sOfZU5NjjsNn7BZ2t9h2tly3Hnoocexw2e7+SbO07SsmWY09Fjj0Om71f\ngNlq31Euthx7KnLscdjs/QrPVvuO8rHl2FORY4/DZu9XeLbad5SPLceeihx7HDZ7v5Js7TtKyZZj\nT0WOPQ6bvd9IbO072g5bjj0VOfY4bPZ+I7C172hbbDn2VOTY47DZ+43A1r6jbbHl2FORY4/DZu83\nAlv7jrbFlmNPRY49Dpu93whs7TvaFluOPRU59jhs9n4jsLdPnx8a0epoQ7Ll2FORY4/DZu8nxy72\niAw5djn2GGz2fiOw5djbYsuxpyLHHofN3i8zu891XyqsJ/YhGXLsXWlV2kUVhgz9MrL7XkO9snpi\nZ2DIscuxx2Cz98vI7nsN9crqiZ2BIccuxx6Dzd4vI7vvNdQrqyd2BoYce1dalXZRhSFDvxXZOa6h\nHrie2AMx5Ni70qq0iyoMGfqtwM61H2nQemIPyJBjl2OPwWbvtwI7136kQeuJPSBDjl2OPQabvd8K\n7Fz7kQatJ/aADDn2rrQq7aIKQ4Z+B7CH3I80QD2xR2aEduxmdqeZfd/MzpvZp3M853WRY4/DZu+3\ngD30fqQNj5aaXaVjN7N1AD8A8OsAXgDwHQD3uPuzqcfIsVfMZu+3gD30fqQNj5aaXaVjN7NfA/An\n7v4b868fAAB3/1zqMXLsFbPZ+y1gD70facOjpWbX6thvAfD8NV+/MD+WL3Lscdjs/dbL7UfawGib\nZFfr2JeJme2Y2b6Z7V++fLnfg+XY47DJ+5Xcj5R8tM2ya3Xsw6sYOfY4bPJ+Ja+VTj7aZtm1OvYj\nmL15ejuAFzF78/Red38m9Rg59orZ5P1KXiudfLTNsqt07O7+CoCPA3gUwHMAvrpoUV8pcuxx2ET9\nol0rnWi0Yi/JCO3Y3f0f3f0X3f0X3P2zOZ7zusixx2GT9It4rXSS0YrdgxHWsa8SOfaK2ST9Il4r\nnWS0YvdghHXsq0SOvWI2Sb+I10onGa3YPRhhHfsokWOPw66wXy3XSq9wtGIfkhHasQ8eOfY47Mr6\n1XSt9MpGK3YGhhy7HHsMdmX9arpWemWjFTsDQ45djj0Gu7J+NV0rvbLRip2BIcfelValXVRhWLhf\n7ddKDzxasQdiyLF3pVVpF1UYFuyXy6VrtGKPyZBjl2OPwQ7aL5dL12jFHpMhxy7HHoMdtF8ul74C\nuiqG2OOz5dhTkWOPww7Qb0iX3vhoxR6ZIcfelValXVRhOEK/ofcdbXi0YhdgyLHLscdgk+872vBo\nxS7AkGOXY4/BJt93dAEaAPVoxS7AkGPvSqvSLqowzNiv1L6jDYxW7JHZcuypyLHHYZPvO0o+WrEL\nsOXYU5Fjj8Mm33eUfLRiF2DLsacixx6HTb7vaCokoxW7AFuOPRU59jhs8n1HiUYrdhC2HHsqcuxx\n2OT7jpKMVuxAbDn2VOTY47DJ9x0lGa3Ygdhy7KnIscdhk+87WpLN3q9Vthx7KnLscdgZGRH3HSUZ\nrdiB2HLsqcixx2FnZGyfPl8KzT5asQOx5dhTkWOPw5Zjr5oh9vhsOfZU5NjjsOXYq2aIPT5bjj0V\nOfY47BUZOa6hrtGKXSNbjj0VOfY47BUYufYj1WjFrpEtx56KHHsc9gqMXPuRarRi18iWY09Fjj0O\newVGrv1INVqxa2TLsacixx6HfQBjyP1IGx+t2JWyq3PsZvZ5M/uemX3XzL5uZm/OdWLXRY49DnsB\nI5dL12iHY4g9Prs6x25mdwD4lru/YmZ/BgDu/scHPU6OvWL2AkYul67RDscQe3x21Y7dzH4bwO+4\n+4cOuq8ce8XsBYxcLn0F9OAp7WKHjthlGDU49o8B+GbG53s9cuxx2HPGkC698dEOyhB7fHZIx25m\nj5vZf3Tc7r7mPg8CeAXA3oLn2TGzfTPbv3z5cr+zlGOPw97ext72dFCX3vBoqfu1yq7OsQOAmZ0F\n8PsAbnf3l5d5jBx7xezNTUxwkbZeVBfL0K9VdnWO3czuBPAQgPe6+9K/hsuxV8w2wxqu0tYryWbv\n1yq7Rsf+RQA3AXjMzJ4ysy8d8vm6I8dehN3l0bGxwVIvHJu9X6vskI59Udz97e5+yt1/ZX77g1wn\ndl3k2Ednpz6Tvrc9ZagXks3er1V2lY59lcixx2enPpNOUi8km71fq+zqHPuqkWOPz059Jp2kXkg2\ne79W2TU69nEixz46O/WZdJJ6Idns/Vpll3DscPfRb1tbW94r06n7sWPusx96s9uxY+67u93Hp9P+\nj8l1vEL29OhZ38QFN1zxTVzw6dGzPt19gqVeNWz2fq2yFzGm035LIYD9ZdbYIwP9vMgf9+6vU8dX\neUyu4xWx93Avduw9eHn+P4VLmGDHHsFHcYShXnXsMRhij88+6Hu5U4dj15ung7G1oXQcNnu/Vtl6\n8zQVvXk6GFsbSsdhs/drla03T1PRm6dZ2NpQOjabvV+r7Or+QGm06A+UDs3WhtLx2ez9WmXrD5RS\nkWM/NFsbSsdns/drlS3Hnooc+6HZ2lA6Ppu9X6tsOfZU5Nh7sbWhdJ1s9n6tsuXYU5FjX5o99CYY\nDY9Wjl3s7Aw5djn2pdhDb4LR8GgHZ7P3a5Utx56KHPvS7KE3wWh4tIOz2fu1ypZjT0WO/YbjpTbB\naGC0xdjs/Vply7GnIsd+XUpugkE+2rAulqFfq2w59lTk2K9LyU0wyEcb1sUy9GuVLceeihz7dSm5\nCQb5aIuy2fu1ypZjT6Vhx97nM+mlfWFlow3HZu/XKluOPZVGHXvf67uU9oUVjTYkm71fq2w59lQa\ndewRr5VOMtqQbPZ+rbLl2FNp1LFHvFY6yWhDstn7tcqWY0+lAcdey7XSKxxtNWz2fq2y5dhTIXfs\nNV0rvbLRVsVm79cqW449FXLHXtO10isbbVVs9n6tsuXYUyF37DVdK72y0VbFZu/XKluOPRVyx17T\ntdIrG21VbPZ+rbLl2FMhd+zbp8+XQodktMpm79cqW449FTn2odAhGa2y2fu1yq7WsZvZ/QD+HMAJ\nd3/poPvLsV8fOXaxx2KIPT67SsduZqcA3AHgR4d9rmSIHHvt+5EGHm31bPZ+rbJrdexfAPApAMP9\nzCNx7Lk+rx7VFzL4UM1W7DEZIR27md0N4DZ3v8/MLgI4M4iKIXHsuVx6VF/I4EM1W7HHZBRz7Gb2\nOIC3dHzrQQCfAXCHu//PQQu7me0A2AGAjY2NrUtdLVMhcey5XPoK6KoYrbLZ+7XKDunY3f397v7O\nN94A/BDA2wA8PV/UTwJ40sy6fgjA3R929zPufubEiRPLNwGqdOxDuvSovpDBh2q2Yo/JCOfY3f3f\n3f3n3H3i7hMALwD4VXf/SbazezWVOfa97emgLj2qL2TwoZqt2GMyQjr2655Ijv019gQXm/SFDP00\nW7HHZIT+HHvfsH+OfQ1Xm/SFQ4edzd6vVXZIxx4iQR17l0fHxkazvpChn2Yr9piMcI591AR07KnP\npO9tT5v1hQz9NFuxx2SEd+x9wuDYU59Jb9kXMvTTbMUekyHHHsyxpz6T3rIvHDrsbPZ+rbLl2FMp\n7Nj7fCa9ZV/I0E+zFXtMhhx7V0YQZ32v79KyL2Top9mKPSZDjr2QY+97fZeWfSFDP81W7DEZcuyF\nHHvf67u07AuHDjubvV+rbDn2VEZy7Dmu79KyL2Top9mKPSZDjr0rGcVZrmult+wLGfpptmKPyZBj\nH9ix17TvaFRfyNBPsxV7TIYc+8COvaZ9R0uy2fuVZLP3a5Utx55KZsde+76jUX0hQz/NVuwxGXLs\nXVlBajHsOxrVFzL002zFHpMhx57JsTPsOxrVFzL002zFHpMhx57JsTPsO1qSzd6vJJu9X6tsOfZU\nMjp2hn1Ho/pChn6ardhjMoZy7HD30W9bW1veK9Op+/Hj7rMferPb8ePuu7vdx6fT5GOmu0/0eqq+\nxxegszFKstn7abZij8mYTvsthQD2l1ljD7xDiIXdfTaBzU13s9m/r04kdXzB9/o+VUY0BZu9n2Yr\n9piMPll2Ya/DsSuKoihkjl1RFEVZOlrYFUVRyKKFXVEUhSxa2BVFUciihV1RFIUsRT4VY2aXAXT8\ngW343AzgpdInMWJa6wuocyuptfOmu5846E5FFvZaY2b7y3zUiCWt9QXUuZWwd5aKURRFIYsWdkVR\nFLJoYe+Xh0ufwMhprS+gzq2EurMcu6IoCln0G7uiKApZtLCvEDO738zczG4ufS5Dx8w+b2bfM7Pv\nmtnXzezNpc9pqJjZnWb2fTM7b2afLn0+Q8fMTpnZP5vZs2b2jJndV/qcxoiZrZvZv5nZN0qfy1DR\nwt4zZnYKwB0AflT6XEbKYwDe6e6/DOAHAB4ofD6DxMzWAfwVgN8E8A4A95jZO8qe1eB5BcD97v4O\nAO8B8EcNdAaA+wA8V/okhowW9v75AoBPAWjizQl3/yd3f2X+5b8CuHHDWI68G8B5d/+hu/8fgK8A\nuLvwOQ0ad/+xuz85/++fYrbY3VL2rIaNmZ0E8AEAXy59LkNGC3uPmNndAF5096dLn0uhfAzAN0uf\nxEC5BcDz13z9AsgXuWtjZhMA7wLw7bJnMnj+ArNfzHrsNFpfjpQ+gWgxs8cBvKXjWw8C+AxmGoYq\nizq7+z/M7/MgZv/XfW/Mc1OGj5m9CcDfA/iEu/9v6fMZKmZ2F4D/cvdzZva+0uczZLSwvyHu/v6u\n42b2SwDeBuBpm207fhLAk2b2bnf/yYinmD2pzq/GzM4CuAvA7c77+dgXAZy65uuT82PUMbOjmC3q\ne+7+tdLnM3BuBfBBM9sG8DMAftbMpu7+4cLnlT36HPuKMbOLAM64e40XElo6ZnYngIcAvNfdL5c+\nn6FiZkcwe3P4dswW9O8AuNfdnyl6YgPGZr+h/A2A/3b3T5Q+nzEz/439k+5+V+lzGSJy7MpB+SKA\nmwA8ZmZPmdmXSp/QEJm/QfxxAI9i9ibiV5kX9XluBfARALfNX9un5r/NKpVHv7EriqKQRb+xK4qi\nkEULu6IoClm0sCuKopBFC7uiKApZtLAriqKQRQu7oigKWbSwK4qikEULu6IoCln+H2jeZ7pvtXxh\nAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x11705de90>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch 10000\n",
      "Hypothesis [[ 0.99773693  0.00226305]\n",
      " [ 0.00290443  0.99709558]\n",
      " [ 0.00295531  0.99704474]\n",
      " [ 0.99804318  0.00195681]]\n",
      "w1=[[-6.62694883  7.52302551]\n",
      " [ 6.91208267 -7.39292049]]\n",
      "b1=[ 3.32245088  3.76204109]\n",
      "w2=[[ 6.63464451 -6.49259472]\n",
      " [ 6.40471601 -6.61061907]]\n",
      "b2=[-9.65064335  9.65065002]\n",
      "cost (ce)=0.0100926\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXYAAAD8CAYAAABjAo9vAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAGZtJREFUeJztnX+IZedZx7/Pzu6gQyP9Y1dasjv3VosLoYp1l9ISoSEp\nMW5Do+IfJmlhKXRwayGFlNI0f/gDgpRKqlClJPUP5Q6UohYhWGKDLSSipbsxqSZpS2xmsoktbhC0\n1D9Mdh//mLtkJ3PeO/ecOee8z/t9v1+47O7ZOfdznudM3p185sz7mLtDURRF4cmh3BegKIqi9Bst\n7IqiKGTRwq4oikIWLeyKoihk0cKuKIpCFi3siqIoZNHCriiKQhYt7IqiKGTRwq4oikKWwzmgR48e\n9el0mgOtKIpSbC5cuPCKux/b7+OyLOzT6RTnz5/PgVYURSk2Zra9zMdJxSiKopBFC7uiKApZtLAr\niqKQRQu7oigKWbSwK4qikKWchX1zE5hOgUOHdn7d3Fx8vMs5fR1nZ7PXp96Kvc85fSIGibuP/jp1\n6pS3ymzmvrbmDrz+WltzP3eu+fhs1v6cvo6zs9nrU2/F3uec2bnHe0PMZu2WQgDnl1ljLcdovNOn\nT3ur59inU2C74fHNlRXg8uW9xyeTnV/bnNPXcXY2e33qrdj7nDNduYjty8d7QUwmwNbW3uOpmNkF\ndz+978cVsbAfOrTzj9yyMdv5NUNt9Gz2+nKy2esjYR/CZXgLi70IYQZcubL0Wy29sJfh2NfXm4+v\nrKQ/vu05fR1nZ7PXp96Kfc05m7gTU7yAQ7iMKV7AJu7E+sp/9IZIHT9oyljYz5xpPn7yZPrj257T\n13F2Nnt96q3Y82ye/ANs4GFsYwrHIWxjig08jLeftL4QyeMHTRkqRo49Dpu9PvVW7HnaunQ5djn2\nctns9eVks9dXGLutS++AkGNvTIHOrng2e33qbZXsPly6HHvbyLHHYbPXp95Wx948M+vFpcuxy7GX\ny2avT72tjj3FVrbbKscuxx6DzV5fTjZ7fUHZh3Al222VY29KYGdHy2avT72lZTd5dKyvZ72tcuxN\nCersqNns9am3lOzUM+mbZ2ZZb6scuxx7DDZ7feotJTv1THru2yrHLsceg81eX042e30Z2aln0nPf\nVjn2plTsC7Ox2etTb4tnt3kmPfdtlWNvSqW+UB6YlM1e3wjstvu7jHVbHzjzBNbw413H1/BjPHDm\nieaTDpqDDMzo+mo9aGMy2b1D/dXXykrz8cmk/Tl9HWdns9en3hbNnqxcDFf2ZLKzhs1wp0/wghsu\n+wQv+Ax3zv9y+WDsQRtmtgLgPICX3f32RR8rx14wm72+nGz2+kZg97lXel8xA64gsYa1lOw5HPs9\nAJ7r8f1ejxx7HDZ7feptMeyh90rvs7yxJXsvC7uZHQfwfgBf7OP99kSOPQ6bvT71tgj2GHulty3j\nppMvpz362A+yL+Nr9nsB+CsApwDcBOCR/T5ejr1gNnt96m0R7LYufYyyJysX0x49BYnq2M3sdgBn\n3P2jZnYTgE94g2M3sw0AGwCwvr5+arvpaf1U5NjjsNnry8lmr69H9hh7pbd+L1zBFTT4mB4fZB/T\nsd8I4ANmtgXgSwBuNrPZGz/I3R9y99PufvrYsWPtCHLscdjs9am34di59kpv/V6Ja8rxIPuBF3Z3\nv8/dj7v7FMBvAfgHd//gga/s2sixx2Gz16fehmL35dL7LDvp0k/+ZXtIZMd+9QU5dn42e33qbSh2\nXy69z7KTLr0LJKpj7xI9x14wm72+nGz2+jqw+3LpHdDpc1IuvQsksGMfPnLscdjs9am32dhDuvQ+\ny0669ECbxZSxsMuxx2Gz16feZmEPPXe0S9mtXXoXSAmOfdmXHHvBbPb61Nss7Ihlt3bpcuxy7MWy\n2evLyWavbwF76LmjC9AAmstu7dI7QeTY9yawL6Rls9en3mZhRyy7tUsP5Nj3/ZJ+iFdrFTObua+u\n7v5fmNVV93Pnmo/PZu3P6es4O5u9PvV2UPbsyNndauPIWffZLHvZa6uv7vq7tdVXfXbu8eF7O5sN\nomIOD/PPxQBxb/5z6niXc/o6zs4eg1ErewxGJvYm7sKGvRv/O192tjHFhj0MzP+cq+y7sQn4Y7gf\nv4sXsY51vIgH/PdxN35ynN4OkDIcu4ZZx2Gz16feDsbIOVB6UdlbmBYzzVrDrIFB/0Wsls1eX042\neX05B0qnsnAIRp8QoJmhb542JOg3gqjZ7PWpt70wog2UXlQ24zTrMhZ2/YBSHDZ7fertgd8r4kDp\nTkMwxuitfkAJe19Bf9iCms1en3p74PcKOVC6yxCMUX4KatJqKYR+QAntzukr7Gz2+nKySeoLOVC6\nyxCM3uBy7M2RY4/DZq9PvW31XsUMlO4yBEOOfeDIscdhs9en3i59TlEDpbsMwZBjl2Ovhs1en3q7\n9DlFDZSOel/l2FtcJ4mrDMlmry8nu7D6ihooPQa8C0OOvSFErrIYNnt96m3jOcUPlI56X+XYG0Li\nKotis9en3u4JxUDpqPdVjj2TtCP3pCEZtbKD1kcxUDrqfZVjb3GdhbnKotjs9eVkB62PYqB0n/A+\nGXLsDSnQVRbPZq+v8t7SDpSOel/l2BtSmKukYLPXV3FvqQdKR72vcuzBvFmtbPb6Ku5txNb25tKj\n3lc59hbXGdRVUrDZ68vJ1kDpvX/Xl0vvAh+DIcfekMCukpbNXl8FvW3y6PQDpaPeVzn2hgR1ldRs\n9vrIe5t6Jn3zzCxrax8488SwLj3qfZVjD+bNamWz10fe29Qz6blb65PJsC496n2VY29xnewuNieb\nvb6cbM0dzQMH8t1XOfaG1Opio/pChvqIequ5o0vC5dgzRY49Dpu9PpLe5pw7mnr2fOPkN/LNHY16\nX6M6dgAnAHwdwLMAngFwz37nyLEXzGavj6S3OeeOLnr2PNvc0aj3NapjN7O3Aniruz9pZtcBuADg\n19z92dQ5cuwFs9nry8kmmTsack/0nOwSHbu7/8Ddn5z//kcAngNw/UHfd1fk2OOw2esrsLfR5o6G\n3BM96n0twbGb2RTAOwF8s8/3lWMPxGavr7De5pw7WtSe6FHva1THfvUF4E3Y0TC/kfj7DQDnAZxf\nX19v5ZXk2AOx2esrrLc5544WtSd61Psa1bEDgJkdAfAIgEfd/cH9Pl6OvWA2e3052YXNHS1qT/Sc\n7BIdu5kZgD8H8Nwyi3qnyLHHYbPXF7i30eaOFrUnetT7Gtix3wjgQwBuNrOn5q9+xZEcexw2e31B\ne5tz7ijFnuhB72t4x97mpefYC2az1xe0tznnjlLsiR70voZ27G0jx14wm72+nOygc0cp9kTPyS7R\nsY8SOfY4bPb6AvQ22txRij3RA9zX5N8NkDIWdjn2OGz2+jL3NufcUeo90aP+NyPHHsyb1cpmry9z\nb7UnOud9lWNvihx7HDZ7fTnZmeeOUu+JnpMtx56IHHscNnt9I7Ejzh1l6W04thx7InLscdjs9Y3A\nzjl3NPlMOvue6FH/m5FjD+bNamWz1zcCO+fc0eQz6SS9DcmWY09Ejj0Om72+Edg5544mn0kn6W1I\nthx7InLscdjs9fXMjjZ3NPlMeoG9LYYtx56IHHscNnt9PbIjzh1NPpNeWG+LYsuxJyLHHofNXl+P\n7KhzRxl6WxRbjj0ROfY4bPb6emRr7qjY+zLk2BtSq7OL6gsZ6uvI1tzRAxxnZ8uxJyLHHofNXl8H\ntuaOHvA4O1uOPRE59jhs9vo6sDV3lPO+yrG3jBx7wWz2+jqwNXdU7M4MOfaG1OrsovpChvr2YWvu\n6ADH2dly7InIscdhs9e3gD30XumaO0rKlmNPRI49Dpu9vgXsrCpWc0fLZcuxJyLHHofNXt8C9tB7\npWvuKClbjj0ROfY4bPb6Es+kj7FXuuaOkrLl2BORY4/DJq8v517pmjtKypZjT0SOPQ6bvL6se6VP\nXHNHGdly7InIscdhk9eXda90zR3lZMuxJyLHHodNVF+4vdI1d5STLceeiBx7HDZJfSH3StfcUU62\nHHsicuxx2CT1hdwrnaS3YrdgyLG3uE52Z5eTTVJfyL3SSXordguGHHtDanV2UX1h0PqK2Su9wN6K\nfUBGZMduZreZ2XfN7Hkz+1Qf77krcuxx2IXVV9Re6YX1VuweGFEdO4AVAP8O4GcArAJ4GsANi86R\nYy+YXVh9Re2VXlhvxe6BEdWxm9l7APyeu//K/M/3zf/B+MPUOXLsBbMLq6+ovdIL663YPTACO/br\nAVy85s8vzY/1Fzn2OOzA9RW/V3rg3oo9ECOyY18mZrZhZufN7PylS5fanSzHHocdtL6+XHqXsjV3\nVOzOjMCO/T0AHr3mz/cBuG/ROXLsBbOD1teXS++kSTV3VOzOnzyTVkshRnTshwF8D8AtAF4G8C0A\nd7n7M6lz5NgLZgetry+X3gGtuaNid2dEdezu/hqAjwF4FMBzAL68aFHvFDn2OOwA9Q3p0jtpUs0d\nFbsrI7Jjd/e/c/efc/efdfcH+njPXZFjj8Mmnzu6qOzB90pn/9yplV2iY+/ykmMvmE0+d3RR2YPv\nlc7+uVMru0TH3iVy7AWzyeeOLkAPv1c6++dOrewSHfsokWOPwyafO7qobJbeij0yu1THPnjk2OOw\nyeeOZt0rnf1zp1a2HHsicuxx2OxzR3Pulc7+uVMrW449ETn2OGz2uaM590pn/9yplS3Hnogcexw2\n+9zRnHuls3/u1MqWY09Ejj0Om33uaM690tk/d2ply7EnIsceh80+d5Skt2IHYsuxJyLHHofNPnd0\nDHhOhtjjs+XYE5Fjj8NmnztaYG/FDs6WY09Ejj0Om33uaGG9FbsAthx7InLscdjsc0cL663YBbDl\n2BORY4/DZp872ic8IkPs8dly7InIscdhs88dDdxbsQtly7EnIsceh80+dzRobyk+d2ply7EnIsce\nh80+dzRobyk+d2ply7EnIsceh80+d7QLvCSG2OOz5dgTkWOPw2afOxqgt4MyxB6fLceeiBx7HDb7\n3NGoLpbhc6dWthx7InLscdjsc0ejuliG+mply7EnIsceh80+d3QRHBi8t4MzxB6fLceeiBx7FnaV\nc0ejuliG+mply7EnIsc+OrvauaNRXSxDfbWy5dgTkWMfnV3t3NGoLpahvlrZcuyJyLGPzq527mgq\nuV3s0BE7D0OOvSG1Orue2Zo7usxFybGLPQBDjr0htTq7HtmaO7rk8dwulqG+Wtly7InIsQ/G1tzR\nQPeVvb5a2XLsicixD8bW3NFAbPb6amXLsScix94LW3NHD3A8t4tlqK9WdmmO3cw+a2bfMbNvm9lX\nzOzNfV3YrsixH5ituaMHPJ7bxTLUVyu7NMcO4FYAh+e//wyAzyxznhz7+GzNHS3gvrLXVyu7ZMdu\nZr8O4Dfd/e79PlaOfXy25o4WwGavr1Z24Y79wwC+2uP7vR459lZszR0d4HhuF8tQX63siI7dzB4z\ns39reN1xzcfcD+A1AJsL3mfDzM6b2flLly61u0o59qXZQ++VTj93NKqLZaivVnZpjn2ucc4C+CcA\na8ueI8c+HDurLmSYOxrVxTLUVyu7NMduZrcBeBDAe9196S/D5diHYw+9Vzr93NGcbPb6amUX6Ng/\nD+A6AF8zs6fM7AsHfL/myLHvOZ5rr3T6uaNRXSxDfbWyIzr2RXH3t7v7CXf/xfnrt/u6sF2RY9+V\nnHul088djepiGeqrlV2iY+/ykmM/2PGse6VPnHvuaFQXy1BfrezSHHvXyLEfLFn3SmefO5qTzV5f\nrewCHfs4kWPfjci5Vzr73NGoLpahvlrZGRz7vl/SD/FqrWJmM/fV1d3/C7O66n7uXPPx2az9OX0d\n75k9O3J2t/I4ctZn5x7vFb22+uqu42urryaPz2Yd7kfQ3oZjs9dXK3sRYzYbRMWU8RU7sNOKpj+n\njnc5p6/jPb3XJu7Chr3hm6T2MP4Rv9wb+kY8gYf8I5hgC4YrmGALD/lH8Gf4aOPxu6/+DFrhvQ3L\nHoMh9vjs/f6u55Th2KdTYHt77/GVFeDy5b3HJ5OdX9uc09fxHtnTlYvYvnx8UPRk5SVsXT7R4gSO\n3oZks9dXK3sRYzIBtrb2Hk9kWcdexsJe6TdPxxiCoWEXgdjs9dXK1jdPE6ngm6e5hmBo2EUgNnt9\ntbJL+wGl0UL+A0pjDMHQsIsC2Oz11crWDyglQv4DSmMMwdCwiwLY7PXVytYPKCVC7tjHGIKhYRcF\nsNnrq5Utx54IkWPPNQRDwy4KYLPXVytbjj0REsfel0vPukFX0N5SsNnrq5Utx54IiWPvy6Vn3aAr\naG8p2Oz11cqWY0+ExLH35dIXoQffoCtobynY7PXVypZjT6RAxz6kS1+ElgcumM1eX61sOfZECnPs\nQw+UTj6TfuYJeeCS2ez11cqWY0+kMMc+uLJLPZOe2xcy+NCoLpahvlrZcuyJFObYBx8onXomPbcv\nHDrsbPb6amXLsScS1LFnGyideiY9ty9k8KFRXSxDfbWy5dgTCejYxxgo3Xp/l9y+kMGHRnWxDPXV\nypZjTySgYx9joHTr/V1y+0IGH6reij0mQ469xXWSDJQOuVc6uw/NyWavr1a2HHsimR17m2fS+0SH\n3Cud3Yeqt2KPyZBjb8gI3qzt/i5d0EXtlc7uQ9VbscdkyLHn8WZt93fpgi5qr3R2H6reij0mQ469\nxXWyzx3tEx6RUSubvb5a2XLsiYzk2IuZOxrVFzL4UPVW7DEZcuwN6dGbFTV3NKovZPCh6q3YYzLk\n2If1ZkXNHY3qCxl8qHor9piMyI7dzO4F8EcAjrn7K/t9fETHXtTc0S7wkhi1stnrq5VdomM3sxMA\nbgXw4kHfK5meHXvxc0ej+kIGH6reij0mI7Bj/xyATwIY7p+8Hh07xdzRqL6QwYeqt2KPyYjo2AHc\nAeBP5r/fAnB0mfNyOnaKuaNRfSFDfeqt2GMycjl2M3sMwFsa/up+AJ8GcKu7/7eZbQE47QnHbmYb\nADYAYH19/dT29vby//r06Ngp5o4uggPDstl9aE42e321siM6dnd/n7u/440vAN8H8DYAT88X9eMA\nnjSzpn8E4O4Puftpdz997NixpQsB0Nmx084djeoLGepTb8UekxHNsbv7v7r7T7v71N2nAF4C8Evu\n/sPeru5qOjh26rmjUX0hQ33qrdhjMiI69mtfCObYqeeORvWFDPWpt2KPyYj8HHvbjPEcO/Xc0VRy\n+8Khw85mr69WdkTHHiItn0mnnzsa1Rcy1Kfeij0mI5pjHzUtn0mnnzsa1Rcy1Kfeij0mI7pjb/Pq\ny7FXO3c0qi9kqE+9FXtMhhz73uusdu5oTjZ7fTnZ7PXVypZjT6TlM+n0c0ej+kKG+tRbscdk1OzY\n2z6TTj93NKovZKhPvRV7TEbNjn2MkadFzR2N6gsZ6lNvxR6TUbNjH2PkaVFzR3Oy2evLyWavr1a2\nHHtzxhh5WtTc0ai+kKE+9VbsMRk1O/Y+R55SzB2N6gsZ6lNvxR6TIcfek9JimDsa1Rcy1Kfeij0m\nQ459+fennzuak81eX042e321suXYm9OnY6eYOxrVFzLUp96KPSZDjn1vFikt6rmjUX0hQ33qrdhj\nMuTY2ykt6rmjUX0hQ33qrdhjMuTYl39/+rmjOdns9eVks9dXK1uOvTldHDu1s4vqCxnqU2/FHpMx\nkGPf90v6IV5tVcxs5r62tvv/YNbW3M+daz4+m3U4qa/jsxk3m70+9VbsMRmz2SAqZt8PiLCw+3yd\nnkzczXZ+vdqP1PFOJ/V1nJ3NXp96K/aYjBZZdmEvwrEriqIoZI5dURRFWT5a2BVFUciihV1RFIUs\nWtgVRVHIooVdURSFLFmeijGzSwC2RwcfPEcBvJL7IkZMbfUCqrmWlFrzxN2P7fdBWRb2UmNm55d5\n1IgltdULqOZawl6zVIyiKApZtLAriqKQRQt7uzyU+wJGTm31Aqq5llDXLMeuKIpCFn3FriiKQhYt\n7B1iZveamZvZ0dzXMnTM7LNm9h0z+7aZfcXM3pz7moaKmd1mZt81s+fN7FO5r2fomNkJM/u6mT1r\nZs+Y2T25r2mMmNmKmf2LmT2S+1qGihb2ljGzEwBuBfBi7msZKV8D8A53/wUA3wNwX+brGSRmtgLg\nTwH8KoAbANxpZjfkvarB8xqAe939BgDvBvA7FdQMAPcAeC73RQwZLezt8zkAnwRQxTcn3P3v3f21\n+R//GcDxnNczYN4F4Hl3/767/x+ALwG4I/M1DRp3/4G7Pzn//Y+ws9hdn/eqho2ZHQfwfgBfzH0t\nQ0YLe4uY2R0AXnb3p3NfS6Z8GMBXc1/EQLkewMVr/vwSyBe5a2NmUwDvBPDNvFcyeP4YO1+YLT9o\ntMAczn0B0WJmjwF4S8Nf3Q/g09jRMFRZVLO7/+38Y+7Hzv+6b455bcrwMbM3AfhrAB939//JfT1D\nxcxuB/Cf7n7BzG7KfT1DRgv7G+Lu72s6bmY/D+BtAJ62nanjxwE8aWbvcvcfjniJvSdV89WY2VkA\ntwO4xXmfj30ZwIlr/nx8fow6ZnYEO4v6prv/Te7rGTg3AviAmZ0B8BMAfsrMZu7+wczX1Xv0HHvH\nmNkWgNPuXuJGQkvHzG4D8CCA97r7pdzXM1TM7DB2vjl8C3YW9G8BuMvdn8l6YQPGdr5C+QsA/+Xu\nH899PWNm/hX7J9z99tzXMkTk2JX98nkA1wH4mpk9ZWZfyH1BQ2T+DeKPAXgUO99E/DLzoj7PjQA+\nBODm+b19av7VrFJ49BW7oigKWfQVu6IoClm0sCuKopBFC7uiKApZtLAriqKQRQu7oigKWbSwK4qi\nkEULu6IoClm0sCuKopDl/wHvRmmnLOs5DwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x116f30e50>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch 20000\n",
      "Hypothesis [[  9.98954773e-01   1.04520307e-03]\n",
      " [  1.35455513e-03   9.98645484e-01]\n",
      " [  1.37042650e-03   9.98629570e-01]\n",
      " [  9.99092221e-01   9.07784502e-04]]\n",
      "w1=[[-7.04857349  7.84673071]\n",
      " [ 7.33061361 -7.6883769 ]]\n",
      "b1=[ 3.53246331  3.89587522]\n",
      "w2=[[ 7.35947943 -7.21742964]\n",
      " [ 7.14059544 -7.34649324]]\n",
      "b2=[-10.74944305  10.7494421 ]\n",
      "cost (ce)=0.00468077\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXYAAAD8CAYAAABjAo9vAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAGZJJREFUeJztnX+IZedZx7/Pzu6gQyP9Y1dasjv3VosLoYp1l9ISoSEp\nMW5Do+IfJmlhKXRwtZBCSmmaP/wBQUolVahQkvqHcgdKUYsQLDHBFhLR0t2YVJO0JTYz2cQWNwha\n6h8mu49/zF2ykznvnXvOnHPe5/2+3y9cdufMPfdznudM3p185s77mLtDURRF4cmh3BegKIqi9Bst\n7IqiKGTRwq4oikIWLeyKoihk0cKuKIpCFi3siqIoZNHCriiKQhYt7IqiKGTRwq4oikKWwzmgR48e\n9el0mgOtKIpSbC5cuPCqux/b73lZFvbpdIrz58/nQCuKohQbM9te5nlSMYqiKGTRwq4oikIWLeyK\noihk0cKuKIpCFi3siqIoZClnYd/cBKZT4NChnT83Nxcf73JOX8fZ2ez1qbdi73NOn4hB4u6jP06d\nOuWtMpu5r625A2881tbcz51rPj6btT+nr+PsbPb61Fux9zlndu6J3hCzWbulEMD5ZdZYyzEa7/Tp\n097qfezTKbDd8PbNlRXg8uW9xyeTnT/bnNPXcXY2e33qrdj7nDNduYjty8d7QUwmwNbW3uOpmNkF\ndz+97/OKWNgPHdr5R27ZmO38maE2ejZ7fTnZ7PWRsA/hMryFxV6EMAOuXFn6pZZe2Mtw7OvrzcdX\nVtLPb3tOX8fZ2ez1qbdiX3POJu7EFC/iEC5jihexiTuxvvIfvSFSxw+aMhb2M2eaj588mX5+23P6\nOs7OZq9PvRV7ns2Tf4gNPIxtTOE4hG1MsYGH8c6T1hciefygKUPFyLHHYbPXp96KPU9bly7HLsde\nLpu9vpxs9voKY7d16R0QcuyNKdDZFc9mr0+9rZLdh0uXY28bOfY4bPb61Nvq2JtnZr24dDl2OfZy\n2ez1qbfVsafYynZb5djl2GOw2evLyWavLyj7EK5ku61y7E0J7Oxo2ez1qbe07CaPjvX1rLdVjr0p\nQZ0dNZu9PvWWkp16T/rmmVnW2yrHLsceg81en3pLyU69Jz33bZVjl2OPwWavLyebvb6M7NR70nPf\nVjn2plTsC7Ox2etTb4tnt3lPeu7bKsfelEp9oTwwKZu9vhHYbfd3Geu2PnDmSazhx7uOr+HHeODM\nk80nHTQHGZjR9dF60MZksnuH+quPlZXm45NJ+3P6Os7OZq9PvS2aPVm5GK7syWRnDZvhTp/gRTdc\n9gle9BnunH9y+WDsQRtmtgLgPIBX3P32Rc+VYy+YzV5fTjZ7fSOw+9wrva+YAVeQWMNaSvYcjv0e\nAM/3+HpvRI49Dpu9PvW2GPbQe6X3Wd7Ykr2Xhd3MjgP4IIAv9fF6eyLHHofNXp96WwR7jL3S25Zx\n08lX0h597DeyL+Nr9nsA+CsApwDcBOCR/Z4vx14wm70+9bYIdluXPkbZk5WLaY+egkR17GZ2O4Az\n7v47ZnYTgE96g2M3sw0AGwCwvr5+arvp3fqpyLHHYbPXl5PNXl+P7DH2Sm/9WriCK2jwMT2+kX1M\nx34jgA+Z2RaALwO42cxmb36Suz/k7qfd/fSxY8faEeTY47DZ61Nvw7Fz7ZXe+rUS15TjjewHXtjd\n/T53P+7uUwC/BeAf3P3DB76yayPHHofNXp96G4rdl0vvs+ykSz/5l+0hkR371Qfk2PnZ7PWpt6HY\nfbn0PstOuvQukKiOvUv0PvaC2ez15WSz19eB3ZdL74BOn5Ny6V0ggR378JFjj8Nmr0+9zcYe0qX3\nWXbSpQfaLKaMhV2OPQ6bvT71Ngt76LmjXcpu7dK7QEpw7Ms+5NgLZrPXp95mYUcsu7VLl2OXYy+W\nzV5fTjZ7fQvYQ88dXYAG0Fx2a5feCSLHvjeBfSEtm70+9TYLO2LZrV16IMe+77f0Qzxaq5jZzH11\ndff/wqyuup8713x8Nmt/Tl/H2dns9am3g7JnR87uVhtHzrrPZtnLXlt9bdfn1lZf89m5J4bv7Ww2\niIo5PMw/FwPEvfnj1PEu5/R1nJ09BqNW9hiMTOxN3IUNey/+d77sbGOKDXsYmH+cq+y7sQn447gf\nv4eXsI51vIQH/A9wN35ynN4OkDIcu4ZZx2Gz16feDsbIOVB6UdlbmBYzzVrDrIFB/0Wsls1eX042\neX05B0qnsnAIRp8QoJmhH542JOgPgqjZ7PWpt70wog2UXlQ24zTrMhZ2/YJSHDZ7fertgV8r4kDp\nTkMwxuitfkEJex9Bf9mCms1en3p74NcKOVC6yxCMUX4LatJqKYR+QQntzukr7Gz2+nKySeoLOVC6\nyxCM3uBy7M2RY4/DZq9PvW31WsUMlO4yBEOOfeDIscdhs9en3i59TlEDpbsMwZBjl2Ovhs1en3q7\n9DlFDZSOel/l2FtcJ4mrDMlmry8nu7D6ihooPQa8C0OOvSFErrIYNnt96m3jOcUPlI56X+XYG0Li\nKotis9en3u4JxUDpqPdVjj2TtCP3pCEZtbKD1kcxUDrqfZVjb3GdhbnKotjs9eVkB62PYqB0n/A+\nGXLsDSnQVRbPZq+v8t7SDpSOel/l2BtSmKukYLPXV3FvqQdKR72vcuzBvFmtbPb6Ku5txNb25tKj\n3lc59hbXGdRVUrDZ68vJ1kDpvZ/ry6V3gY/BkGNvSGBXSctmr6+C3jZ5dPqB0lHvqxx7Q4K6Smo2\ne33kvU29J33zzCzrnugPnHlyWJce9b7KsQfzZrWy2esj723qPekR9kQf1KVHva9y7C2uk93F5mSz\n15eTTT53NOue6Knkvq9y7A2p1cVG9YUM9RH1Ntrc0ax7oke9r3LsDanVxUb1hQz1kfQ259zRkHui\nR72vUR07gBMAvg7gOQDPArhnv3Pk2Atms9dH0tucc0dD7oke9b5Gdexm9nYAb3f3p8zsOgAXAPya\nuz+XOkeOvWA2e3052SRzR0PuiZ6TXaJjd/cfuPtT87//CMDzAK4/6Ovuihx7HDZ7fQX2Ntrc0ZB7\noke9ryU4djObAng3gG/2+bpy7IHY7PUV1tucc0eL2hM96n2N6tivPgC8BTsa5jcSn98AcB7A+fX1\n9VZeSY49EJu9vsJ6m3PuaFF7oke9r1EdOwCY2REAjwB41N0f3O/5cuwFs9nry8kubO5oUXui52SX\n6NjNzAD8OYDnl1nUO0WOPQ6bvb7AvY02d7SoPdGj3tfAjv1GAB8BcLOZPT1/9CuO5NjjsNnrC9rb\nnHNHKfZED3pfwzv2Ng+9j71gNnt9QXubc+4oxZ7oQe9raMfeNnLsBbPZ68vJDjp3lGJP9JzsEh37\nKJFjj8Nmry9Ab6PNHaXYEz3AfU1+boCUsbDLscdhs9eXubc5545S74ke9b8ZOfZg3qxWNnt9mXub\nszzqPdGj/jcjx97iOtmdXU42e3052Znnjl5By//O+oQDvGw59kTk2OOw2esbiR1x7ihLb8Ox5dgT\nkWOPw2avbwR21LmjDL0NyZZjT0SOPQ6bvb4R2JHnjpbe25BsOfZE5NjjsNnrG4GtuaOVseXYE5Fj\nj8Nmr69ntuaOLnNR5Gw59kTk2OOw2evrka25o0seZ2fLsScixx6HzV5fj2zNHeW8r3LsicixF8xm\nr69HtuaOir0vQ469IbU6u6i+kKG+jmzNHT3AcXa2HHsicuxx2Oz1dWBr7ugBj7Oz5dgTkWOPw2av\nrwNbc0c576sce8vIsRfMZq+vA1tzR8XuzJBjb0itzi6qL2Sobx+25o4OcJydLceeiBx7HDZ7fQvY\nQ++VrrmjpGw59kTk2OOw2etbwM6qYjV3tFy2HHsicuxx2Oz1LWAPvVe65o6SsuXYE5Fjj8Nmry/x\nnvQx9krX3FFSthx7InLscdjk9eXcK11zR0nZcuyJyLHHYZPXl3Wv9Ilr7igjW449ETn2OGzy+rLu\nla65o5xsOfZE5NjjsInqC7dXuuaOcrLl2BORY4/DJqkv5F7pmjvKyZZjT0SOPQ6bpL6Qe6WT9Fbs\nFgw59hbXye7scrJJ6gu5VzpJb8VuwZBjb0itzi6qLwxaXzF7pRfYW7EPyIjs2M3sNjP7rpm9YGaf\n7uM1d0WOPQ67sPqK2iu9sN6K3QMjqmMHsALg3wH8DIBVAM8AuGHROXLsBbMLq6+ovdIL663YPTCi\nOnYzex+A33f3X5l/fN/8H4w/Sp0jx14wu7D6itorvbDeit0DI7Bjvx7AxWs+fnl+rL/IscdhB66v\n+L3SA/dW7IEYkR37MjGzDTM7b2bnL1261O5kOfY47KD19eXSu5StuaNid2YEduzvA/DoNR/fB+C+\nRefIsRfMDlpfXy69kybV3FGxO3/xTFothRjRsR8G8D0AtwB4BcC3ANzl7s+mzpFjL5gdtL6+XHoH\ntOaOit2dEdWxu/vrAD4O4FEAzwP4yqJFvVPk2OOwA9Q3pEvvpEk1d1TsrozIjt3d/87df87df9bd\nH+jjNXdFjj0Om3zu6KKyB98rnf1rp1Z2iY69y0OOvWA2+dzRRWUPvlc6+9dOrewSHXuXyLEXzCaf\nO7oAPfxe6exfO7WyS3Tso0SOPQ6bfO7oorJZeiv2yOxSHfvgkWOPwyafO5p1r3T2r51a2XLsicix\nx2Gzzx3NuVc6+9dOrWw59kTk2OOw2eeO5twrnf1rp1a2HHsicuxx2OxzR3Pulc7+tVMrW449ETn2\nOGz2uaM590pn/9qplS3Hnogcexw2+9xRkt6KHYgtx56IHHscNvvc0THgORlij8+WY09Ejj0Om33u\naIG9FTs4W449ETn2OGz2uaOF9VbsAthy7InIscdhs88dLay3YhfAlmNPRI49Dpt97mif8IgMscdn\ny7EnIsceh80+dzRwb8UulC3Hnogcexw2+9zRoL2l+NqplS3Hnogcexw2+9zRoL2l+NqplS3Hnogc\nexw2+9zRLvCSGGKPz5ZjT0SOPQ6bfe5ogN4OyhB7fLYceyJy7HHY7HNHo7pYhq+dWtly7InIscdh\ns88djepiGeqrlS3Hnogcexw2+9zRRXBg8N4OzhB7fLYceyJy7FnYVc4djepiGeqrlS3Hnogc++js\naueORnWxDPXVypZjT0SOfXR2tXNHo7pYhvpqZcuxJyLHPjq72rmjqeR2sUNH7DwMOfaG1OrsemZr\n7ugyFyXHLvYADDn2htTq7Hpka+7oksdzu1iG+mply7EnIsc+GFtzRwPdV/b6amXLsScixz4YW3NH\nA7HZ66uVLceeiBx7L2zNHT3A8dwulqG+WtmlOXYz+5yZfcfMvm1mXzWzt/Z1Ybsix35gtuaOHvB4\nbhfLUF+t7NIcO4BbARye//2zAD67zHly7OOzNXe0gPvKXl+t7JIdu5n9OoDfdPe793uuHPv4bM0d\nLYDNXl+t7MId+0cBfK3H13sjcuyt2Jo7OsDx3C6Wob5a2REdu5k9bmb/1vC445rn3A/gdQCbC15n\nw8zOm9n5S5cutbtKOfal2UPvlU4/dzSqi2Wor1Z2aY59rnHOAvgnAGvLniPHPhw7qy5kmDsa1cUy\n1FcruzTHbma3AXgQwPvdfelvw+XYh2MPvVc6/dzRnGz2+mplF+jYvwDgOgCPmdnTZvbFA75ec+TY\n9xzPtVc6/dzRqC6Wob5a2REd+6K4+zvd/YS7/+L88dt9XdiuyLHvSs690unnjkZ1sQz11cou0bF3\necixH+x41r3SJ849dzSqi2Wor1Z2aY69a+TYD5ase6Wzzx3NyWavr1Z2gY59nMix70bk3Cudfe5o\nVBfLUF+t7AyOfd9v6Yd4tFYxs5n76uru/4VZXXU/d675+GzW/py+jvfMnh05u1t5HDnrs3NPjFL2\n2upru46vrb7ms1mH+xG0t+HY7PXVyl7EmM0GUTFlfMcO7LSi6ePU8S7n9HW8p9faxF3YsDf9kNQe\nxj/ilwcv+0Y8iYf8Y5hgC4YrmGALD/nHcPfV30ErvLdh2WMwxB6fvd/nek4Zjn06Bba39x5fWQEu\nX957fDLZ+bPNOX0d75E9XbmI7cvHc6AxWXkZW5dPDAvJ2NuQbPb6amUvYkwmwNbW3uOJLOvYy1jY\nK/3hacghGCS9Dclmr69Wtn54mkgFPzwtZghGgb0ths1eX63s0n5BabSQ/4LSGEMwUht0bZz8hgZK\nR2Gz11crW7+glAj5LyiNMQRj0QZdGigdhM1eX61s/YJSIuSOfYwhGBp2UQCbvb5a2XLsiRA59lxD\nMDTsogA2e321suXYEyFx7H259KwbdAXtLQWbvb5a2XLsiZA49r5cetYNuoL2loLNXl+tbDn2REgc\ne18ufRF68A26gvaWgs1eX61sOfZECnTsQ7r0RWh54ILZ7PXVypZjT6Qwxz70QOnk0OgzT8oDl8xm\nr69Wthx7IoU59sGVXeo96bl9IYMPjepiGeqrlS3Hnkhhjn3wgdI593Fh96E52ez11cqWY08kqGPP\nNlA65z4u7D40qotlqK9Wthx7IgEd+xgDpZMuPec+Luw+NKqLZaivVrYceyIBHfsYA6UX7e8S0hcy\n+FD1VuwxGXLsLa6TZKB06/1dcvvCocPOZq+vVrYceyKZHXub96SPsid6VF/I4EPVW7HHZMixN2QE\nb9Z2f5cu6NYuPaovZPCh6q3YYzLk2PN4s7b7u3RBt3bpUX0hgw9Vb8UekyHH3uI6C5s7WtRe6ew+\nNCebvb5a2XLsiYzk2HPNHS1qr3R2H6reij0mQ469IT16s5xzR4vaK53dh6q3Yo/JkGMf1pvlnjtK\n4QsZfKh6K/aYjMiO3czuBfDHAI65+6v7PT+iYy9q7mgXeEmMWtns9dXKLtGxm9kJALcCeOmgr5VM\nz469+LmjUX0hgw9Vb8UekxHYsX8ewKcADPdPXo+OnWLuaFRfyOBD1Vuxx2REdOwA7gDwp/O/bwE4\nusx5OR07xdzRqL6QoT71VuwxGbkcu5k9DuBtDZ+6H8BnANzq7v9tZlsATnvCsZvZBoANAFhfXz+1\nvb29/L8+PTp2irmji+DAsGx2H5qTzV5freyIjt3dP+Du73rzA8D3AbwDwDPzRf04gKfMrOkfAbj7\nQ+5+2t1PHzt2bOlCAHR27LRzR6P6Qob61Fuxx2REc+zu/q/u/tPuPnX3KYCXAfySu/+wt6u7mg6O\nnXruaFRfyFCfeiv2mIyIjv3aB4I5duq5o1F9IUN96q3YYzIiv4+9bcZ4Hzv13NFUcvvCocPOZq+v\nVnZExx4iLd+TTj93NKovZKhPvRV7TEY0xz5qWr4nnX7uaFRfyFCfeiv2mIzojr3Noy/HXu3c0ai+\nkKE+9VbsMRly7Huvs9q5oznZ7PXlZLPXVytbjj2Rlu9Jp587GtUXMtSn3oo9JqNmx972Pen0c0ej\n+kKG+tRbscdk1OzYxxh5WtTc0ai+kKE+9VbsMRk1O/YxRp4WNXc0J5u9vpxs9vpqZcuxN2eMkadF\nzR2N6gsZ6lNvxR6TUbNj73PkKcXc0ai+kKE+9VbsMRly7D0pLYa5o1F9IUN96q3YYzLk2Jd/ffq5\noznZ7PXlZLPXVytbjr05fTp2irmjUX0hQ33qrdhjMuTY92aR0qKeOxrVFzLUp96KPSZDjr2d0qKe\nOxrVFzLUp96KPSZDjn3516efO5qTzV5fTjZ7fbWy5dib08WxUzu7qL6QoT71VuwxGQM59n2/pR/i\n0VbFzGbua2u7/w9mbc393Lnm47NZh5P6Oj6bcbPZ61NvxR6TMZsNomL2fUKEhd3n6/Rk4m628+fV\nfqSOdzqpr+PsbPb61Fuxx2S0yLILexGOXVEURSFz7IqiKMry0cKuKIpCFi3siqIoZNHCriiKQhYt\n7IqiKGTJ8q4YM7sEYHt08MFzFMCruS9ixNRWL6Caa0mpNU/c/dh+T8qysJcaMzu/zFuNWFJbvYBq\nriXsNUvFKIqikEULu6IoClm0sLfLQ7kvYOTUVi+gmmsJdc1y7IqiKGTRd+yKoihk0cLeIWZ2r5m5\nmR3NfS1Dx8w+Z2bfMbNvm9lXzeytua9pqJjZbWb2XTN7wcw+nft6ho6ZnTCzr5vZc2b2rJndk/ua\nxoiZrZjZv5jZI7mvZahoYW8ZMzsB4FYAL+W+lpHyGIB3ufsvAPgegPsyX88gMbMVAH8G4FcB3ADg\nTjO7Ie9VDZ7XAdzr7jcAeC+A362gZgC4B8DzuS9iyGhhb5/PA/gUgCp+OOHuf+/ur88//GcAx3Ne\nz4B5D4AX3P377v5/AL4M4I7M1zRo3P0H7v7U/O8/ws5id33eqxo2ZnYcwAcBfCn3tQwZLewtYmZ3\nAHjF3Z/JfS2Z8lEAX8t9EQPlegAXr/n4ZZAvctfGzKYA3g3gm3mvZPD8CXa+MVt+0GiBOZz7AqLF\nzB4H8LaGT90P4DPY0TBUWVSzu//t/Dn3Y+d/3TfHvDZl+JjZWwD8NYBPuPv/5L6eoWJmtwP4T3e/\nYGY35b6eIaOF/U1x9w80HTeznwfwDgDP2M7U8eMAnjKz97j7D0e8xN6TqvlqzOwsgNsB3OK87499\nBcCJaz4+Pj9GHTM7gp1FfdPd/yb39QycGwF8yMzOAPgJAD9lZjN3/3Dm6+o9eh97x5jZFoDT7l7i\nRkJLx8xuA/AggPe7+6Xc1zNUzOwwdn44fAt2FvRvAbjL3Z/NemEDxna+Q/kLAP/l7p/IfT1jZv4d\n+yfd/fbc1zJE5NiV/fIFANcBeMzMnjazL+a+oCEy/wHxxwE8ip0fIn6FeVGf50YAHwFw8/zePj3/\nblYpPPqOXVEUhSz6jl1RFIUsWtgVRVHIooVdURSFLFrYFUVRyKKFXVEUhSxa2BVFUciihV1RFIUs\nWtgVRVHI8v9pDGugnzvKEgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x11709cf90>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "#!/usr/bin/env python\n",
    "#[2017-6-30]tensorflow解异或问题：https://martin-thoma.com/tf-xor-tutorial/\n",
    "\"\"\"\n",
    "Solve the XOR problem with Tensorflow.\n",
    "The XOR problem is a two-class classification problem. You only have four\n",
    "datapoints, all of which are given during training time. Each datapoint has\n",
    "two features:\n",
    "      x o\n",
    "      o x\n",
    "As you can see, the classifier has to learn a non-linear transformation of\n",
    "the features to find a propper decision boundary.\n",
    "\"\"\"\n",
    "\n",
    "__author__ = \"Martin Thoma\"\n",
    "__email__ = \"info@martin-thoma.de\"\n",
    "\n",
    "import tensorflow as tf\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "from sklearn.preprocessing import OneHotEncoder\n",
    "\n",
    "\n",
    "def trans_for_ohe(labels):\n",
    "    \"\"\"Transform a flat list of labels to what one hot encoder needs.\"\"\"\n",
    "    return np.array(labels).reshape(len(labels), -1)\n",
    "\n",
    "\n",
    "def analyze_classifier(sess, i, w1, b1, w2, b2, XOR_X, XOR_T):\n",
    "    \"\"\"Visualize the classification.\"\"\"\n",
    "    print('\\nEpoch %i' % i)\n",
    "    print('Hypothesis %s' % sess.run(hypothesis,feed_dict={input_: XOR_X,target: XOR_T}))\n",
    "    print('w1=%s' % sess.run(w1))\n",
    "    print('b1=%s' % sess.run(b1))\n",
    "    print('w2=%s' % sess.run(w2))\n",
    "    print('b2=%s' % sess.run(b2))\n",
    "    print('cost (ce)=%s' % sess.run(cross_entropy,feed_dict={input_: XOR_X,target: XOR_T}))\n",
    "    # Visualize classification boundary\n",
    "    xs = np.linspace(-5, 5)\n",
    "    ys = np.linspace(-5, 5)\n",
    "    pred_classes = []\n",
    "    for x in xs:\n",
    "        for y in ys:\n",
    "            pred_class = sess.run(hypothesis,\n",
    "                                  feed_dict={input_: [[x, y]]})\n",
    "            pred_classes.append((x, y, pred_class.argmax()))\n",
    "    xs_p, ys_p = [], []\n",
    "    xs_n, ys_n = [], []\n",
    "    for x, y, c in pred_classes:\n",
    "        if c == 0:\n",
    "            xs_n.append(x)\n",
    "            ys_n.append(y)\n",
    "        else:\n",
    "            xs_p.append(x)\n",
    "            ys_p.append(y)\n",
    "    plt.plot(xs_p, ys_p, 'ro', xs_n, ys_n, 'bo')\n",
    "    plt.show()\n",
    "\n",
    "\n",
    "# The training data\n",
    "XOR_X = [[0, 0], [0, 1], [1, 0], [1, 1]]  # Features\n",
    "XOR_Y = [0, 1, 1, 0]  # Class labels\n",
    "assert len(XOR_X) == len(XOR_Y)  # sanity check\n",
    "\n",
    "# Transform labels to targets\n",
    "enc = OneHotEncoder()\n",
    "enc.fit(trans_for_ohe(XOR_Y))\n",
    "XOR_T = enc.transform(trans_for_ohe(XOR_Y)).toarray()\n",
    "\n",
    "# The network\n",
    "nb_classes = 2\n",
    "input_ = tf.placeholder(tf.float32,shape=[None, len(XOR_X[0])],name=\"input\")\n",
    "target = tf.placeholder(tf.float32,shape=[None, nb_classes],name=\"output\")\n",
    "nb_hidden_nodes = 2\n",
    "# enc = tf.one_hot([0, 1], 2)\n",
    "w1 = tf.Variable(tf.random_uniform([2, nb_hidden_nodes], -1, 1, seed=0),name=\"Weights1\")\n",
    "w2 = tf.Variable(tf.random_uniform([nb_hidden_nodes, nb_classes], -1, 1,seed=0),name=\"Weights2\")\n",
    "b1 = tf.Variable(tf.zeros([nb_hidden_nodes]), name=\"Biases1\")\n",
    "b2 = tf.Variable(tf.zeros([nb_classes]), name=\"Biases2\")\n",
    "activation2 = tf.sigmoid(tf.matmul(input_, w1) + b1)\n",
    "hypothesis = tf.nn.softmax(tf.matmul(activation2, w2) + b2)\n",
    "cross_entropy = -tf.reduce_sum(target * tf.log(hypothesis))\n",
    "train_step = tf.train.GradientDescentOptimizer(0.1).minimize(cross_entropy)\n",
    "\n",
    "# Start training\n",
    "#init = tf.initialize_all_variables()#此方法已于2017-3-2删除\n",
    "init = tf.global_variables_initializer()\n",
    "with tf.Session() as sess:\n",
    "    sess.run(init)\n",
    "\n",
    "    for i in range(20001):\n",
    "        sess.run(train_step, feed_dict={input_: XOR_X, target: XOR_T})\n",
    "\n",
    "        if i % 10000 == 0:\n",
    "            analyze_classifier(sess, i, w1, b1, w2, b2, XOR_X, XOR_T)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 异或难题\n",
    "- minsky：异或门是神经网络的命门——诱发神经网络的第一次衰落\n",
    "## 问题介绍\n",
    "异或逻辑\n",
    "![图](https://aimatters.files.wordpress.com/2015/12/xor-graph.png?w=809![image.png](attachment:image.png))\n",
    "## TensorFlow代码实现\n",
    "以下代码来自：\n",
    "- [Solving XOR with a Neural Network in TensorFlow](https://aimatters.wordpress.com/2016/01/16/solving-xor-with-a-neural-network-in-tensorflow/)\n",
    "## 网络结构图（TensorBoard可视化）：\n",
    "![结构图](https://aimatters.files.wordpress.com/2016/01/tf_graph.png?w=809![image.png](attachment:image.png)\n",
    "### We can see that our inputs x-input and y-input are the starts of the graph, and that they flow through the processes at layer2 and layer3, ultimately being used in the cost function."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#方法二：https://aimatters.wordpress.com/2016/01/16/solving-xor-with-a-neural-network-in-tensorflow/\n",
    "#There is one thing I did notice in putting this together: it’s quite slow. \n",
    "#In Octave, I was able to run 10,000 epochs in about 9.5 seconds. \n",
    "#The Python/NumPy example was able to run in 5.8 seconds. \n",
    "#The above TensorFlow example runs in about 28 seconds on my laptop.\n",
    "import tensorflow as tf\n",
    "import time\n",
    "\n",
    "x_ = tf.placeholder(tf.float32, shape=[4,2], name = 'x-input')\n",
    "y_ = tf.placeholder(tf.float32, shape=[4,1], name = 'y-input')\n",
    "\n",
    "Theta1 = tf.Variable(tf.random_uniform([2,2], -1, 1), name = \"Theta1\")\n",
    "Theta2 = tf.Variable(tf.random_uniform([2,1], -1, 1), name = \"Theta2\")\n",
    "\n",
    "Bias1 = tf.Variable(tf.zeros([2]), name = \"Bias1\")\n",
    "Bias2 = tf.Variable(tf.zeros([1]), name = \"Bias2\")\n",
    "\n",
    "with tf.name_scope(\"layer2\") as scope:\n",
    "    A2 = tf.sigmoid(tf.matmul(x_, Theta1) + Bias1)\n",
    "\n",
    "with tf.name_scope(\"layer3\") as scope:\n",
    "    Hypothesis = tf.sigmoid(tf.matmul(A2, Theta2) + Bias2)\n",
    "\n",
    "with tf.name_scope(\"cost\") as scope:\n",
    "    cost = tf.reduce_mean(( (y_ * tf.log(Hypothesis)) + ((1 - y_) * tf.log(1.0 - Hypothesis)) ) * -1)\n",
    "\n",
    "with tf.name_scope(\"train\") as scope:\n",
    "    train_step = tf.train.GradientDescentOptimizer(0.01).minimize(cost)\n",
    "\n",
    "XOR_X = [[0,0],[0,1],[1,0],[1,1]]\n",
    "XOR_Y = [[0],[1],[1],[0]]\n",
    "\n",
    "#init = tf.initialize_all_variables()\n",
    "init = tf.global_variables_initializer()\n",
    "sess = tf.Session()\n",
    "\n",
    "writer = tf.summary.FileWriter(\"./logs/xor_logs\", sess.graph_def)\n",
    "\n",
    "sess.run(init)\n",
    "\n",
    "t_start = time.clock()\n",
    "for i in range(100000):\n",
    "    sess.run(train_step, feed_dict={x_: XOR_X, y_: XOR_Y})\n",
    "    if i % 1000 == 0:\n",
    "        print('Epoch ', i)\n",
    "        print('Hypothesis ', sess.run(Hypothesis, feed_dict={x_: XOR_X, y_: XOR_Y}))\n",
    "        print('Theta1 ', sess.run(Theta1))\n",
    "        print('Bias1 ', sess.run(Bias1))\n",
    "        print('Theta2 ', sess.run(Theta2))\n",
    "        print('Bias2 ', sess.run(Bias2))\n",
    "        print('cost ', sess.run(cost, feed_dict={x_: XOR_X, y_: XOR_Y}))\n",
    "t_end = time.clock()\n",
    "print('Elapsed time ', t_end - t_start)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# tensorboard使用介绍\n",
    "[参考](http://blog.csdn.net/jerry81333/article/details/53004903)\n",
    "\n",
    "## TensorFlow深度学习笔记 Tensorboard入门\n",
    "转载请注明作者：梦里风林\n",
    "Github工程地址：https://github.com/ahangchen/GDLnotes\n",
    "官方教程：https://www.tensorflow.org/versions/master/how_tos/graph_viz/index.html\n",
    "TensorFlow自带的一个强大的可视化工具\n",
    "\n",
    "### 功能\n",
    "\n",
    "这是TensorFlow在MNIST实验数据上得到Tensorboard结果\n",
    "\n",
    "- Event: 展示训练过程中的统计数据（最值，均值等）变化情况\n",
    "- Image: 展示训练过程中记录的图像\n",
    "- Audio: 展示训练过程中记录的音频\n",
    "- Histogram: 展示训练过程中记录的数据的分布图\n",
    "### 原理\n",
    "\n",
    "- 在运行过程中，记录结构化的数据\n",
    "- 运行一个本地服务器，监听6006端口\n",
    "- 请求时，分析记录的数据，绘制\n",
    "### 实现\n",
    "\n",
    "在构建graph的过程中，记录你想要追踪的Tensor\n",
    "```python\n",
    "with tf.name_scope('output_act'):\n",
    "    hidden = tf.nn.relu6(tf.matmul(reshape, output_weights[0]) + output_biases)\n",
    "    tf.histogram_summary('output_act', hidden)\n",
    "```\n",
    "其中，\n",
    "\n",
    "- histogram_summary用于生成分布图，也可以用scalar_summary记录存数值\n",
    "- 使用scalar_summary的时候，tag和tensor的shape要一致\n",
    "- name_scope可以不写，但是当你需要在Graph中体现tensor之间的包含关系时，就要写了，像下面这样：\n",
    "```python\n",
    "with tf.name_scope('input_cnn_filter'):\n",
    "    with tf.name_scope('input_weight'):\n",
    "        input_weights = tf.Variable(tf.truncated_normal(\n",
    "            [patch_size, patch_size, num_channels, depth], stddev=0.1), name='input_weight')\n",
    "        variable_summaries(input_weights, 'input_cnn_filter/input_weight')\n",
    "    with tf.name_scope('input_biases'):\n",
    "        input_biases = tf.Variable(tf.zeros([depth]), name='input_biases')\n",
    "        variable_summaries(input_weights, 'input_cnn_filter/input_biases')\n",
    "```\n",
    "在Graph中会体现为一个input_cnn_filter，可以点开，里面有weight和biases\n",
    "用summary系列函数记录后，Tensorboard会根据graph中的依赖关系在Graph标签中展示对应的图结构\n",
    "官网封装了一个函数，可以调用来记录很多跟某个Tensor相关的数据：\n",
    "```python\n",
    "def variable_summaries(var, name):\n",
    "    \"\"\"Attach a lot of summaries to a Tensor.\"\"\"\n",
    "    with tf.name_scope('summaries'):\n",
    "        mean = tf.reduce_mean(var)\n",
    "        tf.scalar_summary('mean/' + name, mean)\n",
    "        with tf.name_scope('stddev'):\n",
    "            stddev = tf.sqrt(tf.reduce_sum(tf.square(var - mean)))\n",
    "        tf.scalar_summary('sttdev/' + name, stddev)\n",
    "        tf.scalar_summary('max/' + name, tf.reduce_max(var))\n",
    "        tf.scalar_summary('min/' + name, tf.reduce_min(var))\n",
    "        tf.histogram_summary(name, var)\n",
    "```\n",
    "只有这样记录国max和min的Tensor才会出现在Event里面\n",
    "Graph的最后要写一句这个，给session回调\n",
    "```python \n",
    "merged = tf.merge_all_summaries()\n",
    "```\n",
    "Session 中调用\n",
    "\n",
    "构造两个writer，分别在train和valid的时候写数据：\n",
    "```python\n",
    "train_writer = tf.train.SummaryWriter(summary_dir + '/train',\n",
    "                                              session.graph)\n",
    "valid_writer = tf.train.SummaryWriter(summary_dir + '/valid')\n",
    "```\n",
    "这里的summary_dir存放了运行过程中记录的数据，等下启动服务器要用到\n",
    "构造run_option和run_meta，在每个step运行session时进行设置：\n",
    "summary, _, l, predictions = \n",
    "    session.run([merged, optimizer, loss, train_prediction], options=run_options, feed_dict=feed_dict)\n",
    "注意要把merged拿回来，并且设置options\n",
    "在每次训练时，记一次：\n",
    "train_writer.add_summary(summary, step)\n",
    "在每次验证时，记一次：\n",
    "valid_writer.add_summary(summary, step)\n",
    "达到一定训练次数后，记一次meta做一下标记\n",
    "train_writer.add_run_metadata(run_metadata, 'step%03d' % step)\n",
    "查看可视化结果\n",
    "\n",
    "启动TensorBoard服务器：\n",
    "python安装路径/python TensorFlow安装路径/tensorflow/tensorboard/tensorboard.py --logdir=path/to/log-directory\n",
    "注意这个python必须是安装了TensorFlow的python，tensorboard.py必须制定路径才能被python找到，logdir必须是前面创建两个writer时使用的路径\n",
    "\n",
    "比如我的是：\n",
    "\n",
    "/home/cwh/anaconda2/envs/tensorflow/bin/python /home/cwh/anaconda2/envs/tensorflow/lib/python2.7/site-packages/tensorflow/tensorboard/tensorboard.py --logdir=~/coding/python/GDLnotes/src/convnet/summary\n",
    "使用python\n",
    "\n",
    "然后在浏览器输入 http://127.0.0.1:6006 就可以访问到tensorboard的结果\n",
    "强迫症踩坑后记\n",
    "\n",
    "之前我的cnn代码里有valid_prediction，所以画出来的graph有两条分支，不太清晰，所以只留了train一个分支\n",
    "修改前：\n",
    "\n",
    "\n",
    "多分支graph\n",
    "修改后：\n",
    "\n",
    "\n",
    "单分支graph\n",
    "多用with，进行包裹，这样才好看，正如官网说的，你的summary代码决定了你的图结构\n",
    "不是所有的tensor都有必要记录，但是Variable和placeholder最好都用summary记录一下，也是为了好看\n",
    "由于有了gradient的计算，所以与gradient计算相关的都会被拎出来，下次试一下用其他optimizer\n",
    "我的CNN TensorBoard代码：cnn_board.py\n",
    "\n",
    "参考资料\n",
    "mnist_with_summaries.py\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#tensorboard测试,参考:http://blog.csdn.net/jerry81333/article/details/53004903\n",
    "import tensorflow as tf    \n",
    "import numpy as np    \n",
    "    \n",
    "def add_layer(inputs,in_size,out_size,n_layer,activation_function=None): #activation_function=None线性函数    \n",
    "    layer_name=\"layer%s\" % n_layer    \n",
    "    with tf.name_scope(layer_name):    \n",
    "        with tf.name_scope('weights'):    \n",
    "            Weights = tf.Variable(tf.random_normal([in_size,out_size])) #Weight中都是随机变量    \n",
    "            tf.summary.histogram(layer_name+\"/weights\",Weights) #可视化观看变量    \n",
    "        with tf.name_scope('biases'):    \n",
    "            biases = tf.Variable(tf.zeros([1,out_size])+0.1) #biases推荐初始值不为0    \n",
    "            tf.summary.histogram(layer_name+\"/biases\",biases) #可视化观看变量    \n",
    "        with tf.name_scope('Wx_plus_b'):    \n",
    "            Wx_plus_b = tf.matmul(inputs,Weights)+biases #inputs*Weight+biases    \n",
    "            tf.summary.histogram(layer_name+\"/Wx_plus_b\",Wx_plus_b) #可视化观看变量    \n",
    "        if activation_function is None:    \n",
    "            outputs = Wx_plus_b    \n",
    "        else:    \n",
    "            outputs = activation_function(Wx_plus_b)    \n",
    "        tf.summary.histogram(layer_name+\"/outputs\",outputs) #可视化观看变量    \n",
    "        return outputs    \n",
    "    \n",
    "#创建数据x_data，y_data    \n",
    "x_data = np.linspace(-1,1,300)[:,np.newaxis] #[-1,1]区间，300个单位，np.newaxis增加维度    \n",
    "noise = np.random.normal(0,0.05,x_data.shape) #噪点    \n",
    "y_data = np.square(x_data)-0.5+noise    \n",
    "    \n",
    "with tf.name_scope('inputs'): #结构化    \n",
    "    xs = tf.placeholder(tf.float32,[None,1],name='x_input')    \n",
    "    ys = tf.placeholder(tf.float32,[None,1],name='y_input')    \n",
    "    \n",
    "#三层神经，输入层（1个神经元），隐藏层（10神经元），输出层（1个神经元）    \n",
    "l1 = add_layer(xs,1,10,n_layer=1,activation_function=tf.nn.relu) #隐藏层    \n",
    "prediction = add_layer(l1,10,1,n_layer=2,activation_function=None) #输出层    \n",
    "    \n",
    "#predition值与y_data差别    \n",
    "with tf.name_scope('loss'):    \n",
    "    loss = tf.reduce_mean(tf.reduce_sum(tf.square(ys-prediction),reduction_indices=[1])) #square()平方,sum()求和,mean()平均值    \n",
    "    tf.summary.scalar('loss',loss) #可视化观看常量    \n",
    "with tf.name_scope('train'):    \n",
    "    train_step = tf.train.GradientDescentOptimizer(0.1).minimize(loss) #0.1学习效率,minimize(loss)减小loss误差    \n",
    "    \n",
    "init = tf.initialize_all_variables()    \n",
    "sess = tf.Session()    \n",
    "#合并到Summary中    \n",
    "merged = tf.summary.merge_all()    \n",
    "#选定可视化存储目录    \n",
    "writer = tf.summary.FileWriter(\"Desktop/\",sess.graph)    \n",
    "sess.run(init) #先执行init    \n",
    "    \n",
    "#训练1k次    \n",
    "for i in range(1000):    \n",
    "    sess.run(train_step,feed_dict={xs:x_data,ys:y_data})    \n",
    "    if i%50==0:    \n",
    "        result = sess.run(merged,feed_dict={xs:x_data,ys:y_data}) #merged也是需要run的    \n",
    "        writer.add_summary(result,i) #result是summary类型的，需要放入writer中，i步数（x轴）"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
